package com.mvlens

import com.lmq.Utils.MyUtils.splitName
import com.lmq.Utils.SparkSessionSingleton
import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.functions.col

import scala.util.matching.Regex

object processDataOfAD {
  val dataPath ="D:\\pythonProject\\pythonProject\\MyUtils\\test\\Datadir\\myrevisedV1.csv"
  Logger.getLogger("org.apache.spark")
    .setLevel(Level.WARN)
  val spark =  SparkSessionSingleton.getInstance(null)

  def main(args: Array[String]): Unit = {
    val reads = spark.read.option("header", value = true)


    val frame = reads
      .option("delimiter", ",")
      .csv(dataPath )
    frame.show(false)
    frame.select(
      col("index").as("index"),

      splitName(col("actors")).as("actors"),

      splitName(col("directors")).as("directors")
    ).show(false)




  }

}
